{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "effd40eb-07ec-4504-a1f1-1bd1a99c904b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec  \n",
    "# 词嵌入\n",
    "''' \n",
    "词嵌入工具需要的数据不必携带标签且tokenzier总返回最新的数据集分词器.\n",
    "\n",
    "'''\n",
    "def tokenzier(update_corpus=True):\n",
    "    # 是否需要更新分词器数据集\n",
    "    if update_corpus or not os.path.exists(CORPUS_PATH): \n",
    "        lines = pd.read_csv(RAW_PATH, encoding='utf-8').iloc[:, 0].values\n",
    "        lines = [line.split(';')[-1] for line in lines]\n",
    "        print(f'Saving: {CORPUS_PATH}')\n",
    "        # 纯语料数据写入\n",
    "        with open(CORPUS_PATH, 'w', encoding='utf-8') as file:\n",
    "            file.write(\"\\n\".join(lines))\n",
    "            file.close()\n",
    "    # 打开文件并读取所有行  \n",
    "    with open(CORPUS_PATH, 'r', encoding='utf-8') as file:  \n",
    "        sentences = file.readlines() \n",
    "    # 去除每行末尾的换行符（如果需要）  \n",
    "    sentences = [sentence.strip() for sentence in sentences]  \n",
    "    embed_dim = 10       # 设置词嵌入的维度为10 \n",
    "    # 训练Word2Vec模型  \n",
    "    tokenizer = Word2Vec(sentences, vector_size=embed_dim, window=5, min_count=1, workers=4)    \n",
    "    return tokenizer\n",
    "# Test 分词\n",
    "test_tokenzier = tokenzier(False)\n",
    "print(test_tokenzier)\n",
    "# 要分词的文本  \n",
    "text = 'ᠠᠷᠠᠭ᠎ᠠ'  \n",
    "# 对文本进行分词  \n",
    "encoded_input = test_tokenzier.wv[text]  \n",
    "print(output)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
